#!/usr/bin/python3
# -*- coding: utf-8 -*-

from selenium import webdriver
from selenium.webdriver import ChromeOptions
from time import sleep
from lxml import etree
import csv

"""
#selenium.webdriver.chrome.options.Options
用于设置chrome浏览器，目前使用过的参数：
1、阻止chrome弹窗的出现：'profile.default_content_setting_values' :{'notifications' : 2}
2、不加载图片："profile.managed_default_content_settings.images":2
3、无头浏览器，--headless
"""

class Vip_Product_Spider():
    womenClothURL = 'https://www.vip.com/nav/53174447.html'
    womenClothFile='vip-women-brand.csv'
    manClothURL='https://www.vip.com/nav/53174448.html'
    manClothFile='vip-men-brand.csv'
    # webDriver = '''D:\workspace\PycharmProjects\spider-python\com\wushi\chromedriver.exe'''
    chromeName = 'C:\\MyApps\\chromedriver.exe'

    def __init__(self):
        # 配置chrome浏览器的相关参数
        option = ChromeOptions()
        option.add_experimental_option('prefs', {'profile.managed_default_content_settings.images': 2,
                                                 'profile.default_content_setting_values' :{'notifications' : 2}
                                                 })
        # option.add_argument('--headless')
        # option.add_argument('--disable-gpu')  # 设置无头浏览器
        self.bro=webdriver.Chrome(executable_path = Vip_Product_Spider.chromeName, options = option)
        # Chrome DevTools Protocol ：Evaluates given script in every frame upon creation (before loading frame's scripts).
        # 目的在打开窗口前隐藏window.navigator.webdriver
        self.bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
                Object.defineProperty(navigator, 'webdriver', {
                  get: () => undefined
                })
              """
        })  # 可能失效

    def running(self, startURL, csv_file):
        self.load_brand(startURL,csv_file)
        self.bro.quit()

    def load_brand(self, url,csv_file):
        self.bro.get(url)
        old_brandIndex = 0;
        is_continuse = True
        while is_continuse:
            is_continuse,old_brandIndex = self.parser_data_brand(old_brandIndex,csv_file)
            self.bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')  # 向下拉动一屏
            sleep(2)

    def parser_data_brand(self,old_brandIndex,csv_file) -> bool:
        result = False
        # 通过etree解析页面；
        html=etree.HTML(self.bro.page_source)
        div_list=html.xpath('//*[@id="J-index-channelb-brandlist"]/div')[0:]
        if(old_brandIndex == 0 or (old_brandIndex >0 and div_list.__len__()>old_brandIndex)):
            print('parse the page from {} to {}'.format(old_brandIndex,div_list.__len__()))
            result = True
        else:
            print('the end of the page!!!')
            return result,old_brandIndex

        for div in div_list[old_brandIndex:]:
            # sleep(0.5)
            dic={}
            try:
                dic["title"]=div.xpath('.//a/div[1]/span[3]/text()')[0]
            except:
                dic["title"]=""
            try:
                dic["sale_discount"]=div.xpath('.//a/div[1]/span[1]/text()')[1]
            except:
                dic["sale_discount"]=""
            try:
                dic["img_link"]=div.xpath('.//a/img/@src')[0]
            except:
                dic["img_link"]=""
            try:
                dic["details_link"]="https:"+div.xpath('.//a/@href')[0]
            except:
                dic["details_link"]=""
            with open(csv_file, "a", encoding="utf-8") as f:
                writer = csv.DictWriter(f, dic.keys())
                writer.writerow(dic)
        old_brandIndex = div_list.__len__()
        return result,old_brandIndex

if __name__ == "__main__":
    Spider=Vip_Product_Spider()
    # Spider.running(Vip_Product_Spider.womenClothURL)
    Spider.running(Vip_Product_Spider.manClothURL,Vip_Product_Spider.manClothFile)




